package com.chance.cc.crawler.development.bootstrap.weibo.toRedis;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.utils.RedisReader;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRecordFilter.dateRange;
import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.turnPageItem;

/**
 * @ClassName weibo
 * @Description TODO
 * @Author songding
 * @Date 2021/10/27 9:50
 * @Version 1.0
 *  微博帖子地址列表
 **/
public class WeiboTieUrlListStart {
    public static final String domainId = "weibo";
    public static void main(String[] args) {
        createCommentRecordToRedis();//只采集回复
    }
    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }
    private static void createCommentRecordToRedis(){
        String site = "tie";
        String siteBiz = "simple-id3-20211112";
        String path = "E:\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\weibo\\toRedis\\id3.txt";
       RedisReader redisReader = new RedisReader("192.168.1.211",6379,3);

        try {
            BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(path), "UTF-8"));
            String readLine = null;
            while ((readLine = bufferedReader.readLine()) != null) {
                if(StringUtils.isBlank(readLine)){
                    continue;
                }
                String[] split = readLine.split("\t");
                String url = split[0];
                String mid = split[1];
                System.out.println(url +"****"+mid);

                CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                        .startPageRequest(domainId, turnPageItem)
                        .httpUrl(url)
                        .releaseTime(System.currentTimeMillis())
                        .filter(dateRange)
//                        .addFilterInfo(FilterUtils.redisFilterKeyInfo(StringUtils.joinWith("-","filter",domainId,site,siteBiz,"queue")))
                        .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*100,null))//时间范围
                        //.resultLabelTag(CrawlerEnum.CrawlerDataType.article)// 采集文章
                        //.resultLabelTag(CrawlerEnum.CrawlerDataType.interaction)//采集互动量
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.comment) //采集评论
                        .proxy(proxy)
                        .build();
                requestRecord.tagsCreator().bizTags().addDomain(domainId);
                requestRecord.tagsCreator().bizTags().addSite(site);
                requestRecord.tagsCreator().bizTags().addSiteBiz(siteBiz);
                requestRecord.tagsCreator().bizTags().addCustomKV("mid",mid);
                String requestKey = requestRecord.getRecordKey();

                CrawlerRecord filterRecord = new CrawlerRecord();
                filterRecord.setFilter(dateRange);                              //评论采集指定时间过滤条件
                filterRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(0,new long[]{1633363200000l,System.currentTimeMillis()}));
                requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info",JSON.toJSONString(filterRecord));

                //确保key值唯一
                String shaKey = DigestUtils.sha1Hex(System.currentTimeMillis()+requestKey);
                redisReader.mapPush("crawler-weibo-tie-simple-queue_map",shaKey, JSON.toJSONString(requestRecord));
                redisReader.listPush("crawler-weibo-tie-simple-queue_list",shaKey);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
   /* @Test
    public int tt(){
        int i = 0;
        for (int a=0;a<1000;a++){
             i = new Random().nextInt(100)*1000;
            if (i<30000 || i>60000){
                break;
            }
        }
        return i;
    }*/
}


